{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 扩展库 Gym 的使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "列出所有环境"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>观测空间</th>\n",
       "      <th>动作空间</th>\n",
       "      <th>奖励范围</th>\n",
       "      <th>最大步数</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Copy-v0</th>\n",
       "      <td>Discrete(6)</td>\n",
       "      <td>(Discrete(2), Discrete(2), Discrete(5))</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RepeatCopy-v0</th>\n",
       "      <td>Discrete(6)</td>\n",
       "      <td>(Discrete(2), Discrete(2), Discrete(5))</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ReversedAddition-v0</th>\n",
       "      <td>Discrete(4)</td>\n",
       "      <td>(Discrete(4), Discrete(2), Discrete(3))</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ReversedAddition3-v0</th>\n",
       "      <td>Discrete(4)</td>\n",
       "      <td>(Discrete(4), Discrete(2), Discrete(3))</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DuplicatedInput-v0</th>\n",
       "      <td>Discrete(6)</td>\n",
       "      <td>(Discrete(2), Discrete(2), Discrete(5))</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reverse-v0</th>\n",
       "      <td>Discrete(3)</td>\n",
       "      <td>(Discrete(2), Discrete(2), Discrete(2))</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CartPole-v0</th>\n",
       "      <td>Box(4,)</td>\n",
       "      <td>Discrete(2)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CartPole-v1</th>\n",
       "      <td>Box(4,)</td>\n",
       "      <td>Discrete(2)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MountainCar-v0</th>\n",
       "      <td>Box(2,)</td>\n",
       "      <td>Discrete(3)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MountainCarContinuous-v0</th>\n",
       "      <td>Box(2,)</td>\n",
       "      <td>Box(1,)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pendulum-v0</th>\n",
       "      <td>Box(3,)</td>\n",
       "      <td>Box(1,)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Acrobot-v1</th>\n",
       "      <td>Box(6,)</td>\n",
       "      <td>Discrete(3)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Blackjack-v0</th>\n",
       "      <td>(Discrete(32), Discrete(11), Discrete(2))</td>\n",
       "      <td>Discrete(2)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KellyCoinflip-v0</th>\n",
       "      <td>(Box(1,), Discrete(301))</td>\n",
       "      <td>Discrete(25000)</td>\n",
       "      <td>(0, 250.0)</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KellyCoinflipGeneralized-v0</th>\n",
       "      <td>(Box(1,), Discrete(309), Discrete(309), Discre...</td>\n",
       "      <td>Discrete(20000)</td>\n",
       "      <td>(0, 200.0)</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FrozenLake-v0</th>\n",
       "      <td>Discrete(16)</td>\n",
       "      <td>Discrete(4)</td>\n",
       "      <td>(0, 1)</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FrozenLake8x8-v0</th>\n",
       "      <td>Discrete(64)</td>\n",
       "      <td>Discrete(4)</td>\n",
       "      <td>(0, 1)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CliffWalking-v0</th>\n",
       "      <td>Discrete(48)</td>\n",
       "      <td>Discrete(4)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NChain-v0</th>\n",
       "      <td>Discrete(5)</td>\n",
       "      <td>Discrete(2)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Roulette-v0</th>\n",
       "      <td>Discrete(1)</td>\n",
       "      <td>Discrete(38)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Taxi-v3</th>\n",
       "      <td>Discrete(500)</td>\n",
       "      <td>Discrete(6)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GuessingGame-v0</th>\n",
       "      <td>Discrete(4)</td>\n",
       "      <td>Box(1,)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HotterColder-v0</th>\n",
       "      <td>Discrete(4)</td>\n",
       "      <td>Box(1,)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CubeCrash-v0</th>\n",
       "      <td>Box(40, 32, 3)</td>\n",
       "      <td>Discrete(3)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CubeCrashSparse-v0</th>\n",
       "      <td>Box(40, 32, 3)</td>\n",
       "      <td>Discrete(3)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CubeCrashScreenBecomesBlack-v0</th>\n",
       "      <td>Box(40, 32, 3)</td>\n",
       "      <td>Discrete(3)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MemorizeDigits-v0</th>\n",
       "      <td>Box(24, 32, 3)</td>\n",
       "      <td>Discrete(10)</td>\n",
       "      <td>(-inf, inf)</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                             观测空间  \\\n",
       "Copy-v0                                                               Discrete(6)   \n",
       "RepeatCopy-v0                                                         Discrete(6)   \n",
       "ReversedAddition-v0                                                   Discrete(4)   \n",
       "ReversedAddition3-v0                                                  Discrete(4)   \n",
       "DuplicatedInput-v0                                                    Discrete(6)   \n",
       "Reverse-v0                                                            Discrete(3)   \n",
       "CartPole-v0                                                               Box(4,)   \n",
       "CartPole-v1                                                               Box(4,)   \n",
       "MountainCar-v0                                                            Box(2,)   \n",
       "MountainCarContinuous-v0                                                  Box(2,)   \n",
       "Pendulum-v0                                                               Box(3,)   \n",
       "Acrobot-v1                                                                Box(6,)   \n",
       "Blackjack-v0                            (Discrete(32), Discrete(11), Discrete(2))   \n",
       "KellyCoinflip-v0                                         (Box(1,), Discrete(301))   \n",
       "KellyCoinflipGeneralized-v0     (Box(1,), Discrete(309), Discrete(309), Discre...   \n",
       "FrozenLake-v0                                                        Discrete(16)   \n",
       "FrozenLake8x8-v0                                                     Discrete(64)   \n",
       "CliffWalking-v0                                                      Discrete(48)   \n",
       "NChain-v0                                                             Discrete(5)   \n",
       "Roulette-v0                                                           Discrete(1)   \n",
       "Taxi-v3                                                             Discrete(500)   \n",
       "GuessingGame-v0                                                       Discrete(4)   \n",
       "HotterColder-v0                                                       Discrete(4)   \n",
       "CubeCrash-v0                                                       Box(40, 32, 3)   \n",
       "CubeCrashSparse-v0                                                 Box(40, 32, 3)   \n",
       "CubeCrashScreenBecomesBlack-v0                                     Box(40, 32, 3)   \n",
       "MemorizeDigits-v0                                                  Box(24, 32, 3)   \n",
       "\n",
       "                                                                   动作空间  \\\n",
       "Copy-v0                         (Discrete(2), Discrete(2), Discrete(5))   \n",
       "RepeatCopy-v0                   (Discrete(2), Discrete(2), Discrete(5))   \n",
       "ReversedAddition-v0             (Discrete(4), Discrete(2), Discrete(3))   \n",
       "ReversedAddition3-v0            (Discrete(4), Discrete(2), Discrete(3))   \n",
       "DuplicatedInput-v0              (Discrete(2), Discrete(2), Discrete(5))   \n",
       "Reverse-v0                      (Discrete(2), Discrete(2), Discrete(2))   \n",
       "CartPole-v0                                                 Discrete(2)   \n",
       "CartPole-v1                                                 Discrete(2)   \n",
       "MountainCar-v0                                              Discrete(3)   \n",
       "MountainCarContinuous-v0                                        Box(1,)   \n",
       "Pendulum-v0                                                     Box(1,)   \n",
       "Acrobot-v1                                                  Discrete(3)   \n",
       "Blackjack-v0                                                Discrete(2)   \n",
       "KellyCoinflip-v0                                        Discrete(25000)   \n",
       "KellyCoinflipGeneralized-v0                             Discrete(20000)   \n",
       "FrozenLake-v0                                               Discrete(4)   \n",
       "FrozenLake8x8-v0                                            Discrete(4)   \n",
       "CliffWalking-v0                                             Discrete(4)   \n",
       "NChain-v0                                                   Discrete(2)   \n",
       "Roulette-v0                                                Discrete(38)   \n",
       "Taxi-v3                                                     Discrete(6)   \n",
       "GuessingGame-v0                                                 Box(1,)   \n",
       "HotterColder-v0                                                 Box(1,)   \n",
       "CubeCrash-v0                                                Discrete(3)   \n",
       "CubeCrashSparse-v0                                          Discrete(3)   \n",
       "CubeCrashScreenBecomesBlack-v0                              Discrete(3)   \n",
       "MemorizeDigits-v0                                          Discrete(10)   \n",
       "\n",
       "                                       奖励范围  最大步数  \n",
       "Copy-v0                         (-inf, inf)   200  \n",
       "RepeatCopy-v0                   (-inf, inf)   200  \n",
       "ReversedAddition-v0             (-inf, inf)   200  \n",
       "ReversedAddition3-v0            (-inf, inf)   200  \n",
       "DuplicatedInput-v0              (-inf, inf)   200  \n",
       "Reverse-v0                      (-inf, inf)   200  \n",
       "CartPole-v0                     (-inf, inf)   200  \n",
       "CartPole-v1                     (-inf, inf)   500  \n",
       "MountainCar-v0                  (-inf, inf)   200  \n",
       "MountainCarContinuous-v0        (-inf, inf)   999  \n",
       "Pendulum-v0                     (-inf, inf)   200  \n",
       "Acrobot-v1                      (-inf, inf)   500  \n",
       "Blackjack-v0                    (-inf, inf)  None  \n",
       "KellyCoinflip-v0                 (0, 250.0)  None  \n",
       "KellyCoinflipGeneralized-v0      (0, 200.0)  None  \n",
       "FrozenLake-v0                        (0, 1)   100  \n",
       "FrozenLake8x8-v0                     (0, 1)   200  \n",
       "CliffWalking-v0                 (-inf, inf)  None  \n",
       "NChain-v0                       (-inf, inf)  1000  \n",
       "Roulette-v0                     (-inf, inf)   100  \n",
       "Taxi-v3                         (-inf, inf)   200  \n",
       "GuessingGame-v0                 (-inf, inf)   200  \n",
       "HotterColder-v0                 (-inf, inf)   200  \n",
       "CubeCrash-v0                    (-inf, inf)  None  \n",
       "CubeCrashSparse-v0              (-inf, inf)  None  \n",
       "CubeCrashScreenBecomesBlack-v0  (-inf, inf)  None  \n",
       "MemorizeDigits-v0               (-inf, inf)  None  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "space_names = ['观测空间', '动作空间', '奖励范围', '最大步数']\n",
    "df = pd.DataFrame(columns=space_names)\n",
    "\n",
    "env_specs = gym.envs.registry.all()\n",
    "for env_spec in env_specs:\n",
    "    env_id = env_spec.id\n",
    "    try:\n",
    "        env = gym.make(env_id)\n",
    "        observation_space = env.observation_space\n",
    "        action_space = env.action_space\n",
    "        reward_range = env.reward_range\n",
    "        max_episode_steps = None\n",
    "        if isinstance(env, gym.wrappers.time_limit.TimeLimit):\n",
    "            max_episode_steps = env._max_episode_steps\n",
    "        df.loc[env_id] = [observation_space, action_space, reward_range, max_episode_steps]\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "with pd.option_context('display.max_rows', None):\n",
    "    display(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 小车上山 MountainCar-v0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "环境：Gym库的 MountainCar-v0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "观测空间 = Box(2,)\n",
      "动作空间 = Discrete(3)\n",
      "观测范围 = [-1.2  -0.07] ~ [0.6  0.07]\n",
      "动作数 = 3\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('MountainCar-v0')\n",
    "print('观测空间 = {}'.format(env.observation_space))\n",
    "print('动作空间 = {}'.format(env.action_space))\n",
    "print('观测范围 = {} ~ {}'.format(env.observation_space.low,\n",
    "        env.observation_space.high))\n",
    "print('动作数 = {}'.format(env.action_space.n))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "智能体：一个根据指定确定性策略决定动作并且不学习的智能体"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BespokeAgent:\n",
    "    def __init__(self, env):\n",
    "        pass\n",
    "    \n",
    "    def decide(self, observation): # 决策\n",
    "        position, velocity = observation\n",
    "        lb = min(-0.09 * (position + 0.25) ** 2 + 0.03,\n",
    "                0.3 * (position + 0.9) ** 4 - 0.008)\n",
    "        ub = -0.07 * (position + 0.38) ** 2 + 0.07\n",
    "        if lb < velocity < ub:\n",
    "            action = 2\n",
    "        else:\n",
    "            action = 0\n",
    "        return action # 返回动作\n",
    "\n",
    "    def learn(self, *args): # 学习\n",
    "        pass\n",
    "    \n",
    "agent = BespokeAgent(env)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "智能体与环境交互"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play_montecarlo(env, agent, render=False, train=False):\n",
    "    episode_reward = 0. # 记录回合总奖励，初始化为0\n",
    "    observation = env.reset() # 重置游戏环境，开始新回合\n",
    "    while True: # 不断循环，直到回合结束\n",
    "        if render: # 判断是否显示\n",
    "            env.render() # 显示图形界面，图形界面可以用 env.close() 语句关闭\n",
    "        action = agent.decide(observation)\n",
    "        next_observation, reward, done, _ = env.step(action) # 执行动作\n",
    "        episode_reward += reward # 收集回合奖励\n",
    "        if train: # 判断是否训练智能体\n",
    "            agent.learn(observation, action, reward, done) # 学习\n",
    "        if done: # 回合结束，跳出循环\n",
    "            break\n",
    "        observation = next_observation\n",
    "    return episode_reward # 返回回合总奖励"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "交互1回合，并图形化显示"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "回合奖励 = -105.0\n"
     ]
    }
   ],
   "source": [
    "env.seed(0) # 设置随机数种子,只是为了让结果可以精确复现,一般情况下可删去\n",
    "episode_reward = play_montecarlo(env, agent, render=True)\n",
    "print('回合奖励 = {}'.format(episode_reward))\n",
    "env.close() # 此语句可关闭图形界面"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "评估性能：交互100回合求平均"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "平均回合奖励 = -102.61\n"
     ]
    }
   ],
   "source": [
    "episode_rewards = [play_montecarlo(env, agent) for _ in range(100)]\n",
    "print('平均回合奖励 = {}'.format(np.mean(episode_rewards)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
